/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package csci311;

import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Vector;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;

/**
 *
 * @author Elveslab
 */
public class Indexer {

    private static final String FILE_EXTENSION_FILTER = ".txt";
    protected static File[] files;
    protected ArrayList<Boolean> filesProcessed;
    protected static File INDEX_DIR;
    protected static String FIELD_PATH;
    protected static String FIELD_FILENAME;
    protected static String FIELD_CONTENTS;
    protected static File BACKUP_INDEX_DIR;

    public void createIndex(File INDEX_DIRstr,File BACKUP_INDEX_DIRstr, String FILES_TO_INDEX_DIRECTORY, String FIELD_PATHstr, String FIELD_FILENAMEstr, String FIELD_CONTENTSstr, boolean PERFORM_RECURSIVE_CRAWL) throws CorruptIndexException, LockObtainFailedException, IOException {
        INDEX_DIR = INDEX_DIRstr;
        BACKUP_INDEX_DIR = BACKUP_INDEX_DIRstr;
        FIELD_PATH = FIELD_PATHstr;
        FIELD_FILENAME = FIELD_FILENAMEstr;
        FIELD_CONTENTS = FIELD_CONTENTSstr;

        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_34);
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_34, analyzer);
        IndexWriter indexWriter = new IndexWriter(FSDirectory.open(INDEX_DIR), indexWriterConfig);

        indexWriter.deleteAll();
        indexWriter.commit();


        indexWriter.deleteUnusedFiles();
        indexWriter.optimize();
        indexWriter.close();

        File dir = new File(FILES_TO_INDEX_DIRECTORY);

        FilenameFilter filter = new FilenameFilter() {

            public boolean accept(File dir, String name) {
                return name.endsWith(FILE_EXTENSION_FILTER);
            }
        };
        files = listFilesAsArray(dir, filter, PERFORM_RECURSIVE_CRAWL);

        filesProcessed = new ArrayList();

        for (int i = 0; i < files.length; i++) {
            filesProcessed.add(false);
        }

        //Distribute indexing among dynamic number of threads based upon 1 thread per 5 files
        for (int i = 0; i < files.length; i++) {
            if (i % 5 == 0) {
                Thread t = new Thread(new Runnable() {

                    public void run() {
                        indexDocument();
                    }
                });
                t.start();
            }
        }
        
        //Backup Index
        backupIndex();
    }

    private void indexDocument() {
        try {
            Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_34);
            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_34, analyzer);
            IndexWriter indexWriter = new IndexWriter(FSDirectory.open(INDEX_DIR), indexWriterConfig);

            for (int i = 0; i < files.length; i++) {
                File file = files[i];
                if (file.isFile() && filesProcessed.get(i).equals(false)) {
                    filesProcessed.set(i, true);
                    Document document = new Document();

                    String path = file.getAbsolutePath();
                    String filename = file.getName();

                    path = path.substring(0, path.lastIndexOf(filename));
                    document.add(new Field(FIELD_PATH, path, Field.Store.YES, Field.Index.ANALYZED));

                    document.add(new Field(FIELD_FILENAME, filename, Field.Store.YES, Field.Index.ANALYZED));

                    Reader reader = new FileReader(file);
                    document.add(new Field(FIELD_CONTENTS, reader));

                    indexWriter.addDocument(document, analyzer);
                }
            }

            indexWriter.deleteUnusedFiles();
            indexWriter.optimize();
            indexWriter.close();
        } catch (Exception e) {
        }
    }
    
    private void backupIndex() {
        try {
            Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_34);
            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_34, analyzer);
            IndexWriter indexWriter = new IndexWriter(FSDirectory.open(BACKUP_INDEX_DIR), indexWriterConfig);

            indexWriter.addIndexes(FSDirectory.open(INDEX_DIR));

            indexWriter.deleteUnusedFiles();
            indexWriter.optimize();
            indexWriter.close();
        } catch (Exception e) {
        }
    }

    private static File[] listFilesAsArray(
            File directory,
            FilenameFilter filter,
            boolean recurse) {
        Collection<File> files = listFiles(directory,
                filter, recurse);
//Java4: Collection files = listFiles(directory, filter, recurse);

        File[] arr = new File[files.size()];
        return files.toArray(arr);
    }

    private static Collection<File> listFiles(
            // Java4: public static Collection listFiles(
            File directory,
            FilenameFilter filter,
            boolean recurse) {
        // List of files / directories
        Vector<File> files = new Vector<File>();
// Java4: Vector files = new Vector();

        // Get files / directories in the directory
        File[] entries = directory.listFiles();

        // Go over entries
        for (File entry : entries) {
// Java4: for (int f = 0; f < files.length; f++) {
// Java4: 	File entry = (File) files[f];

            // If there is no filter or the filter accepts the 
            // file / directory, add it to the list
            if (filter == null || filter.accept(directory, entry.getName())) {
                files.add(entry);
            }

            // If the file is a directory and the recurse flag
            // is set, recurse into the directory
            if (recurse && entry.isDirectory()) {
                files.addAll(listFiles(entry, filter, recurse));
            }
        }

        // Return collection of files
        return files;
    }
}
